from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
import os
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.models import load_model
from tensorflow.keras.utils import plot_model
import matplotlib.pyplot as plt
from matplotlib import rcParams, cycler
import glob
from MulticoreTSNE import MulticoreTSNE as TSNE
loaded = np.load('../data.npz')
noisy_data=loaded['test_noisy_data']
target_data=noisy_data[0]
noisy_mixed=[loaded['x_test_noisy']]
test_labels = loaded['test_labels_cat']
noisy_data=np.vstack((noisy_data,noisy_mixed))
model=[]
tf.keras.backend.clear_session()
def salt_and_pepper_mixed(images):
images = images +10*np.random.randint(-1,2, size =images.shape)*np.array([np.random.choice(2, images.shape[1], p=[1-p, p]) for p in np.random.choice(11, images.shape[0])/10])
images = np.clip(images, -1., 1.)
return(images)
def print_imm(imgs,text='',n=20):
m=len(imgs)
plt.figure(figsize=(40, 2 * len(imgs) + 2))
for i in range(n):
for j in range(m):
# display original
ax = plt.subplot(len(imgs), n, i + 1 + j * n)
plt.imshow(imgs[j][i].reshape(28, 28))
if len(text)==m and i==0:
plt.annotate(text[j], xy=(0, 0), xytext=(0, -2))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
def get_predictions_and_eval(noisy_data,file):
tf.keras.backend.clear_session()
model=[]
model = load_model(file)
predictions = []
evaluations = []
for i,noise_level in enumerate(noisy_data):
actual_pred=[]
print("Noise lvl",i*10,"%")
print("predicting")
actual_pred=model.predict(noise_level, verbose=1)
predictions.append(actual_pred)
print("Classification")
class_acu_out=classifier.evaluate(actual_pred, test_labels, verbose=1)[1]
actual_pred=(actual_pred+1)/2
target_data_aux=(target_data+1)/2
actual_pred=np.around(actual_pred,2)
target_data_aux=np.around(target_data_aux,2)
#actual_pred=np.around(actual_pred,2)
#target_data_aux=np.around(target_data,2)
# convert to 0 to 1
#actual_pred=(actual_pred+1)/2
#target_data_aux=(target_data_aux+1)/2
actual_pred=np.around(actual_pred,2)
target_data_aux=np.around(target_data_aux,2)
print("Metrics")
o = tf.keras.metrics.MeanSquaredError()
_ = o.update_state(actual_pred,target_data_aux)
m = tf.keras.metrics.Accuracy()
#_ = m.update_state(np.around(actual_pred,2),np.around(target_data_aux,2))
_ = m.update_state(actual_pred, target_data_aux)
n = tf.keras.metrics.MeanAbsoluteError()
_ = n.update_state(actual_pred,target_data_aux)
p = tf.keras.metrics.BinaryCrossentropy()
_ = p.update_state(actual_pred,target_data_aux)
print("Image Metrics")
arg1=tf.cast(tf.convert_to_tensor(target_data_aux.reshape((28,28,10000))), tf.float32)
arg2=tf.convert_to_tensor(actual_pred.reshape((28,28,10000)))
ssim1 = tf.reduce_mean(tf.image.ssim(arg1, arg2, max_val=1))
psnr1 = tf.reduce_mean(tf.image.psnr(arg1, arg2, max_val=1))
FID= np.around(calculate_fid(actual_pred,target_data_aux),2)
print("TSNE")
embeddings = TSNE(n_jobs=7).fit_transform(actual_pred)
evaluations.append([o.result().numpy(), m.result().numpy(),n.result().numpy(),p.result().numpy(), class_acu_out, ssim1, psnr1,FID,embeddings]) #['loss', 'accuracy', 'mape', 'BinaryCrossentropy']
model=[]
tf.keras.backend.clear_session()
return np.array(predictions),np.array(evaluations)
from numpy import cov
from numpy import trace
from numpy import iscomplexobj
from numpy.random import random
from scipy.linalg import sqrtm
# calculate frechet inception distance
def calculate_fid(act1, act2):
# calculate mean and covariance statistics
mu1, sigma1 = act1.mean(axis=0), cov(act1, rowvar=False)
mu2, sigma2 = act2.mean(axis=0), cov(act2, rowvar=False)
# calculate sum squared difference between means
ssdiff = np.sum((mu1 - mu2)**2.0)
# calculate sqrt of product between cov
covmean = sqrtm(sigma1.dot(sigma2))
# check and correct imaginary numbers from sqrt
if iscomplexobj(covmean):
covmean = covmean.real
# calculate score
fid = ssdiff + trace(sigma1 + sigma2 - 2.0 * covmean)
return fid
Mnist Data with noising¶
label_num=['original']+['Noisy_'+str(i)+'0%' for i in range(1,11)]+['Mixed']
print_imm(noisy_data,label_num)
Training structure (Decision Tree)¶
Nosiy one level¶
Random renoised test DataSet of Mnist, $10k$ examples.
Nosiy multiple levels¶
Test DataSet of Mnist concatenate with diferent noise levels probabilities from $0\%,10\%, \cdots ,100\%$, $11 \times 10000 = 110k$ examples.
First idea¶
Refined $LR1$ and renoising¶
Visual refinment with only usefull results remaining¶
Further tested hypotheses¶
Different types and levels of renoising between autoencoders only while training:¶
- Constant leve: All examples are renoised with the same probabilitie $\in [0\%,10\%, \cdots , 100\%]$
- Mixed: Each examples is independently renoised with a probabilitie $\in [0\%,10\%, \cdots , 100\%]$
- Multi: Only used for Noisy multiple levels renoise each of the $11$ groups of $10000$ examples with one of the probabilities $\in [0\%,10\%, \cdots , 100\%]$
Renoise in evaluation:¶
While evaluating renoise the prediction of the first autonecoder before the prediction of the second autoencoder. The results where worse.
Visual presentation of the best results¶
parent_autoencoder1='../Runs/13_10_2020/Noisy_one_level/001/autoencoder'#[filename for filename in glob.iglob('../Runs/13_10_2020/'+noise_type+'/001/autoencoder', recursive=True)][0]
child_autoencoders1=['../Runs/13_10_2020/Noisy_one_level/001/0/0001/autoencoder',
'../Runs/13_10_2020/Noisy_one_level/001/25/0001/autoencoder',
'../Runs/13_10_2020/Noisy_one_level/001/Mixed/0001/autoencoder']
parent_autoencoder2='../Runs/13_10_2020/Noisy_multiple_level/001/autoencoder'#[filename for filename in glob.iglob('../Runs/13_10_2020/'+noise_type+'/001/autoencoder', recursive=True)][0]
child_autoencoders2=['../Runs/13_10_2020/Noisy_multiple_level/001/0/0001/autoencoder',
'../Runs/13_10_2020/Noisy_multiple_level/001/25/0001/autoencoder',
'../Runs/13_10_2020/Noisy_multiple_level/001/Mixed/0001/autoencoder',
'../Runs/13_10_2020/Noisy_multiple_level/001/Multi/0001/autoencoder']
recalculate=False
if recalculate:
classifier = load_model("../Runs/Classifier/Classifier_model")
predictions1,evaluations1 = get_predictions_and_eval(noisy_data,parent_autoencoder1)
print("autoencoder 001 one lvl done")
predictions2,evaluations2 = get_predictions_and_eval(noisy_data,parent_autoencoder2)
print("autoencoder 001 multi lvl done")
data=[]
data.append(np.array(["autoencoder 001 one lvl",predictions1,evaluations1],dtype=object))
for file in child_autoencoders1:
name=file[19:-12]
predictions,evaluations = get_predictions_and_eval(predictions1,file)
data.append(np.array([name,predictions,evaluations],dtype=object))
print(name, "done")
data.append(np.array(["autoencoder 001 multi lvl",predictions2,evaluations2],dtype=object))
for file in child_autoencoders2:
name=file[19:-12]
predictions,evaluations = get_predictions_and_eval(predictions2,file)
data.append(np.array([name,predictions,evaluations],dtype=object))
print(name, "done")
data = np.array(data)
np.savez_compressed('./Storage/evaluations.npz', data=data)
else:
loaded=np.load('./Storage/evaluations.npz',allow_pickle=True)
data=loaded['data']
for i in [0,2,5,8,10,11]:#range(0,12,2):
if i==0:
print('Input data: Original Mnist')
elif i==11:
print('-'*290)
print('Input data: Mnist with Mixed Noise levels')
else:
print('-'*290)
print('Input data: Mnist with %i0%% Noise' %i)
plot_data=[noisy_data[0],noisy_data[i]]
[plot_data.append(x[i]) for x in data[:,1]]
text=['Target','Input']
[text.append(x) for x in data[:,0]]
print_imm(plot_data,text)
Evaluation comments¶
For the metrics the ouput $y \in [-1,1]$ was trasformed to $\hat{y} \in [0,1]$ and then rounded to two decimal places, since we want to better observe the dieffernce betwen runs instead of the absolute value.
For the classifier the data was not converted.
#['loss', 'accuracy', 'mape', 'BinaryCrossentropy','psnr','ssim']
from matplotlib.pyplot import cm
cmap = plt.cm.tab10#coolwarm
rcParams['axes.prop_cycle'] = cycler(color=cmap(np.linspace(0, 1, len(child_autoencoders2)+1)))
#['loss', 'accuracy', 'mape']
fig=plt.figure(figsize=(45,50));
ax=plt.subplot(4, 2,1);
x_label=[str(i)+'0%' for i in range(0,11)]+['Mixed']
plt.title("Loss over noise $\downarrow$")
pos=0
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
#plt.legend(loc='upper left');
ax=plt.subplot(4, 2,2)
plt.title("accuracy over noise $\\uparrow$")
pos=1
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
#plt.legend(loc='upper left');
plt.legend(bbox_to_anchor=(1, 1));
ax=plt.subplot(4, 2,3)
plt.title("mae over noise $\downarrow$")
pos=2
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
ax=plt.subplot(4, 2,4)
plt.title("binary crossentropy noise $\downarrow$")
pos=3
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
plt.legend(bbox_to_anchor=(1, 1));
ax=plt.subplot(4, 2,5)
plt.title("classifier accuracy $\\uparrow$")
pos=4
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
ax=plt.subplot(4, 2,6)
plt.title("SSIM $\\uparrow$")
pos=5
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
plt.legend(bbox_to_anchor=(1, 1));
ax=plt.subplot(4, 2,7)
plt.title("PSNR $\\uparrow$")
pos=6
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
ax=plt.subplot(4, 2,8)
plt.title("FID $\downarrow$")
pos=7
for i, net in enumerate([x[:,pos] for x in data[:1,2]]):
plt.plot(net,'-x', label=data[i,0])
for i, net in enumerate([x[:,pos] for x in data[1:len(child_autoencoders1)+1,2]]):
plt.plot(net,'-o', label=data[1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+1:len(child_autoencoders1)+2,2]]):
plt.plot(net,'--x', label=data[len(child_autoencoders1)+1+i,0])
for i, net in enumerate([x[:,pos] for x in data[len(child_autoencoders1)+2:,2]]):
plt.plot(net,'--o', label=data[len(child_autoencoders1)+2+i,0])
ax.set_xlabel('Noise level')
plt.xticks(np.arange(12), x_label)
#plt.legend(loc='upper left');
plt.legend(bbox_to_anchor=(1, 1));
plt.show()
Loss(MSE):¶
Vector of $n$ predictions from a sample of $n$ data points.
$Y$ vector of observed values (targets)
$X$ vectror of predicted values
mean squared error:
$$
\mathrm{MSE}=\frac{1}{n} \sum^{n}\left(Y_{i}-X_{i}\right)^{2}
$$
Accuracy:¶
accuracy is the fraction of predictions the model got right. Formally, accuracy has the following definition: $$ \text{Accuracy} =\frac{\text { Number of correct predictions }}{\text { Total number of predictions }}$$
MAE:¶
Mean absolute error (MAE) is a measure of errors between paired observations expressing the same phenomenon. MAE is calculated as: $$ \operatorname{MAE}=\frac{\sum_{i=1}^{n}\left|Y_{i}-X_{i}\right|}{n} $$
Binary crossentropy:¶
$$ \text { Binary crossentropy }=-\frac{1}{\text { n }} \sum^{\text {n }} Y_{i} \cdot \log X_{i}+\left(1-Y_{i}\right) \cdot \log \left(1-X_{i}\right) $$where $x_{i}$ is the $i$ -th value in the model output, $y_{i}$ is the corresponding target value, and output size is the number of values in the model output.
Classifier accuracy:¶
Accuracy of a classifier trained on the "clean" MNIST dataset and evaluated on the predicted outputs of the autoencoders. (See Accuracy above)
SSIM:¶
The structural similarity index measure (SSIM) is a method for predicting the perceived quality of digital television and cinematic pictures, as well as other kinds of digital images and videos. SSIM is used for measuring the similarity between two images. The SSIM index is a full reference metric; in other words, the measurement or prediction of image quality is based on an initial uncompressed or distortion-free image as reference. Wikipedia
Algorithm¶
The SSIM index is calculated on various windows of an image. The measure between two windows $x$ and $y$ of common size $\mathrm{N} \times \mathrm{N}$ ($11 \times 11$ Gaussian filter of width 1.5 is used in tensorflow) is: $$\operatorname{SSIM}(x, y)=\frac{\left(2 \mu_{x} \mu_{y}+c_{1}\right)\left(2 \sigma_{x y}+c_{2}\right)}{\left(\mu_{x}^{2}+\mu_{y}^{2}+c_{1}\right)\left(\sigma_{x}^{2}+\sigma_{y}^{2}+c_{2}\right)}$$ with:
- $\mu_{x}$ the average of $x$;
- $\mu_{y}$ the average of $y$;
- $\sigma_{x}^{2}$ variance of $x_{i}$
- $\sigma_{y}^{2}$ the variance of $y$
- $\sigma_{x y}$ the covariance of $x$ and $y$
- $c_{1}=\left(k_{1} L\right)^{2}, c_{2}=\left(k_{2} L\right)^{2}$ two variables to stabilize the division with weak denominator;
- $L$ the dynamic range of the pixel-values
- $k_{1}=0.01$ and $k_{2}=0.03$ by default.
Formula components¶
The SSIM formula is based on three comparison measurements between the samples of $x$ and $y:$ luminance $(l)$, contrast $(c)$ and structure $(s)$. The individual comparison functions are:
- $l(x, y)=\frac{2 \mu_{x} \mu_{y}+c_{1}}{\mu_{x}^{2}+\mu_{y}^{2}+c_{1}}$
- $c(x, y)=\frac{2 \sigma_{x} \sigma_{y}+c_{2}}{\sigma_{x}^{2}+\sigma_{y}^{2}+c_{2}}$
- $s(x, y)=\frac{\sigma_{x y}+c_{3}}{\sigma_{x} \sigma_{y}+c_{3}}$
with, in addition to above definitions:
- $ c_{3}=\frac{c_{2}}{ 2}$
SSIM is then a weighted combination of those comparative measures: $$\operatorname{SSIM}(x, y)=\left[l(x, y)^{\alpha} \cdot c(x, y)^{\beta} \cdot s(x, y)^{\gamma}\right]$$
Setting the weights $\alpha, \beta, \gamma$ to 1 , the formula can be reduced to the form shown above.
PSNR:¶
Peak signal-to-noise ratio, often abbreviated PSNR, is an engineering term for the ratio between the maximum possible power of a signal and the power of corrupting noise that affects the fidelity of its representation. Wikipedia
Definition¶
PSNR is most easily defined via the mean squared error (MSE). Given a noise-free $m \times n$ monochrome image $I$ and its noisy approximation $\mathrm{K}, \mathrm{MSE}$ is defined as:
$$
\operatorname{MSE}=\frac{1}{m n} \sum_{i=0}^{m-1} \sum_{j=0}^{n-1}[I(i, j)-K(i, j)]^{2}
$$
The PSNR (in $\mathrm{dB}$ ) is defined as:
$$
\begin{aligned}
\operatorname{PSNR} &=10 \cdot \log _{10}\left(\frac{M A X_{I}^{2}}{M S E}\right) \\
&=20 \cdot \log _{10}\left(\frac{M A X_{I}}{\sqrt{M S E}}\right) \\
&=20 \cdot \log _{10}\left(M A X_{I}\right)-10 \cdot \log _{10}(M S E)
\end{aligned}
$$
Here, MAX is the maximum possible pixel value of the image ($1$ in our case).
In tensprflow the implementation uses sets of images instead of single images.
FID:¶
The Frechet Inception Distance score, or FID for short, is a metric that calculates the distance between feature vectors calculated for real and generated images.
The score summarizes how similar the two groups are in terms of statistics on computer vision features of the raw images calculated using the inception v3 model used for image classification. Lower scores indicate the two groups of images are more similar, or have more similar statistics, with a perfect score being 0.0 indicating that the two groups of images are identical.
The FID score is used to evaluate the quality of images generated by generative adversarial networks, and lower scores have been shown to correlate well with higher quality images.(Source)
Equation:¶
$$d^2 = ||\mu_1 – \mu_2||^2 + \operatorname{Tr} \left[ \Sigma_1 + \Sigma_2 – 2*\operatorname{sqrt} \left( \Sigma_1*\Sigma_2 \right) \right]$$The score is referred to as $d^2$, showing that it is a distance and has squared units.
The $\mu_1$ and $\mu_2$ refer to the feature-wise mean of the real and generated images, e.g. $2048$ element vectors where each element is the mean feature observed across the images.
The $\Sigma_1$ and $\Sigma_2$ are the covariance matrix for the real and generated feature vectors.
The $||\mu_1 – \mu_2||^2$ refers to the sum squared difference between the two mean vectors. $\operatorname{Tr}$ refers to the trace linear algebra operation, e.g. the sum of the elements along the main diagonal of the square matrix.
The $\operatorname{sqrt}$ is the square root of the square matrix, given as the product between the two covariance matrices.
TSNE¶
t-distributed stochastic neighbor embedding (t-SNE) of the reconstructed images.
for j in [0,2,5,8,10,11]:#range(12):
fig=plt.figure(figsize=(45,5));
if j==11:
print('-'*290)
print("Mixed")
elif j==0:
print("Noise "+str(j*10)+"%")
else:
print('-'*290)
print("Noise "+str(j*10)+"%")
for i in range(9):
ax=plt.subplot(1, 9,i+1);
plt.title(data[i,0])
embeddings=data[i,2][j,8]
vis_x = embeddings[:, 0]
vis_y = embeddings[:, 1]
plt.scatter(vis_x, vis_y, c=np.argmax(test_labels, axis=1), cmap=plt.cm.get_cmap("jet", 10), marker='.')
plt.colorbar(ticks=range(10))
plt.clim(-0.5, 9.5)
plt.show()
Remarks¶
Training and validation data renoising¶
The test data that was used in validation for the training was renoised in a random mixed form while the input data for training was renoised with an equal noise lvl, with the idea of simulating real world case for the validation. Butin the iteration renoising the prediction of the validation data and the input data was equaly renoised instead of the validation set being randomly renoised.
Accuracy¶
Acc means equal pixels are more but unequal pixels have a higher difference.
!jupyter nbconvert --to html Comparisson_all.ipynb
!jupyter nbconvert --to html --no-input --output-dir='./No_code' Comparisson_all.ipynb